import pickle
import os

# 读取原始数据
def load_raw_data(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        return f.readlines()

# 数据预处理：去除空行、去除多余的空格等
def preprocess_data(raw_data):
    processed_data = []
    for line in raw_data:
        line = line.strip()  # 去除两端的空格
        if line:  # 如果行不为空
            processed_data.append(line)
    return processed_data

# 保存处理过的数据
def save_processed_data(processed_data, output_path):
    with open(output_path, 'wb') as f:
        pickle.dump(processed_data, f)

def main():
    raw_data_path = "data/raw_data.txt"
    processed_data_path = "data/preprocessed_data.pkl"
    
    # 确保 data 文件夹存在
    os.makedirs("data", exist_ok=True)
    
    # 读取原始数据
    raw_data = load_raw_data(raw_data_path)
    
    # 预处理数据
    processed_data = preprocess_data(raw_data)
    
    # 保存预处理数据
    save_processed_data(processed_data, processed_data_path)
    print(f"预处理数据已保存到 {processed_data_path}")

if __name__ == "__main__":
    main()
